diff --git a/drivers/mmc/uniphier-sd.c b/drivers/mmc/uniphier-sd.c
index 40a5c85..701b26f 100644
--- a/drivers/mmc/uniphier-sd.c
+++ b/drivers/mmc/uniphier-sd.c
@@ -571,6 +571,9 @@
 		val = UNIPHIER_SD_CLKCTL_DIV1024;
 
 	tmp = readl(priv->regbase + UNIPHIER_SD_CLKCTL);
+	if (tmp & UNIPHIER_SD_CLKCTL_SCLKEN &&
+	    (tmp & UNIPHIER_SD_CLKCTL_DIV_MASK) == val)
+		return;
 
 	/* stop the clock before changing its rate to avoid a glitch signal */
 	tmp &= ~UNIPHIER_SD_CLKCTL_SCLKEN;
@@ -582,6 +585,8 @@
 
 	tmp |= UNIPHIER_SD_CLKCTL_SCLKEN;
 	writel(tmp, priv->regbase + UNIPHIER_SD_CLKCTL);
+
+	udelay(1000);
 }
 
 static int uniphier_sd_set_ios(struct udevice *dev)
@@ -599,8 +604,6 @@
 	uniphier_sd_set_ddr_mode(priv, mmc);
 	uniphier_sd_set_clk_rate(priv, mmc);
 
-	udelay(1000);
-
 	return 0;
 }